# encoding:utf-8
import sys

sys.path.append('..')
import datetime
import gevent
from gevent import monkey
from source.entity.Base import session
from source.entity.CeilometerMetrics import CeilometerMetrics
import httplib
import json
from decimal import Decimal
from source.confs.constants import Config
from source.auth.identity import Identity
monkey.patch_all()
class MeterManagerParallel:
    '''
    获取CPU、内存、磁盘和网卡监控数据
    '''
    def getSample(self, meter_name, resource_id, token, endpoint, vm_id, retries=5):
        date_now_1 = datetime.datetime.now()
        ret_out = {}
        ret = {}
        headers = {"Content-type": "application/json"}
        headers['X-Auth-Token'] = token
        reqUrl = endpoint + Config.GetMeters_Rest_URL + '/' + str(meter_name) + '?q.field=resource&q.op=eq&q.value=' + str(resource_id) + '&limit=1'

        httpClient = httplib.HTTPConnection(Config.OpenStack_URL, Config.MonitoringService_Port, Config.Request_Timeout)
        try:
            httpClient.request("GET", reqUrl, headers=headers)
            response = httpClient.getresponse()
            status = response.status
            if response.status == 200:
                body = json.loads(response.read())
                # print body
                if body:
                    usage_avg = float(body[0].get('counter_volume'))
                    if meter_name == Config.METER_NAME_MEM:
                        flavor_ram = float(body[0].get('resource_metadata').get('flavor.ram'))
                        usage = Decimal.from_float(usage_avg / flavor_ram * 100).quantize(Decimal('0.00'))
                        ret['timestamp'] = str(body[0].get('timestamp'))
                    else:
                        usage = Decimal.from_float(usage_avg).quantize(Decimal('0.00'))
                    ret[meter_name] = usage
        except Exception, e:
            print e.message
        else:
            ret_out = {vm_id: ret}
        finally:
            if httpClient:
                httpClient.close();
            date_now_2 = datetime.datetime.now()
            print 'vm: %s, meter_name: %s, response.status: %s' % (resource_id, meter_name, status)
            print date_now_2 - date_now_1
        if status == 200:
            return ret_out
        elif status != 200 and retries > 1:
            print '####taotao retries'
            return self.getSample(meter_name, resource_id, token, endpoint, vm_id, retries=retries - 1)
        else:
            date_now_2 = datetime.datetime.now()
            print 'ERROR vm: %s, meter_name: %s, response.status: %s GET Failed' % (resource_id, meter_name, status)
            print date_now_2 - date_now_1
            return None

    '''
    调用虚拟机下的所有网卡
    '''
    def getInterfaces(self, resource_id, token, endpoint, retries=5):
        ret = {}
        nics = []
        headers = {"Content-type": "application/json"}
        headers['X-Auth-Token'] = token
        reqUrl = endpoint + Config.GetServers_S_Rest_URL + '/' +  str(resource_id) + Config.GetInterface_Rest_URL
        httpClient = httplib.HTTPConnection(Config.OpenStack_URL, Config.MonitoringService_Port, Config.Request_Timeout)

        try:
            httpClient.request("GET", reqUrl, headers=headers)
            response = httpClient.getresponse()
            status = response.status
            if response.status == 200:
                body = json.loads(response.read())
                if body:
                    for nic in body.get('interfaceAttachments'):
                        nics.append(nic.get('port_id'))
        except Exception, e:
            print e.message
        else:
            ret = {resource_id: {'interfaces': nics}}
        finally:
            if httpClient:
                httpClient.close();
        if status == 200:
            return ret
        elif status != 200 and retries > 1:
            print '####taotao retries'
            return self.getInterfaces(resource_id, token, endpoint, retries=retries - 1)
        else:
            return None

    '''
        获取虚拟机下的所有磁盘
    '''
    def getVolumes(self, resource_id, token, endpoint, retries=5):
        ret = {}
        volumes = []
        headers = {"Content-type": "application/json"}
        headers['X-Auth-Token'] = token
        reqUrl = endpoint + Config.GetServers_S_Rest_URL + '/' + str(resource_id) + Config.GetVolume_Rest_URL
        httpClient = httplib.HTTPConnection(Config.OpenStack_URL, Config.MonitoringService_Port, Config.Request_Timeout)

        try:
            httpClient.request("GET", reqUrl, headers=headers)
            response = httpClient.getresponse()
            status = response.status
            if response.status == 200:
                body = json.loads(response.read())
                if body:
                    for volume in body.get('volumeAttachments'):
                        volumes.append(volume.get('volumeId'))
        except Exception, e:
            print e.message
        else:
            ret = {resource_id: {'disks': volumes}}
        finally:
            if httpClient:
                httpClient.close();
        if status == 200:
            return ret
        elif status != 200 and retries > 0:
            print '####taotao retries'
            return self.getVolumes( resource_id, token, endpoint, retries=retries - 1)
        else:
            return None


    def getAlarmNums(self, resource_id, retries=5):
        ret = {}
        meter = {}
        httpClient = httplib.HTTPConnection(Config.OpenO_URL, Config.OpenO_Port, Config.Request_Timeout)
        headers = {"Content-type": "application/json"}
        reqUrl = Config.GetCurAlarms_Rest_URL + '?oid=' + resource_id

        try:
            httpClient.request("GET", reqUrl, headers=headers)
            response = httpClient.getresponse()
            status = response.status
            if response.status == 200:
                body = json.loads(response.read())
                meter[Config.METER_ALARM_NUMS] = int(body)
        except Exception, e:
            print e.message
        else:
            ret = { resource_id : meter}
        finally:
            if httpClient:
                httpClient.close()
        if status == 200:
            return ret
        elif status != 200 and retries > 1:
            print '####taotao retries'
            return self.getAlarmNums(resource_id, retries=retries - 1)
        else:
            return None

    def getVms(self, token, endpoint, retries=5):
        resourceIdList = []
        headers = {"Content-type": "application/json"}
        headers['X-Auth-Token'] = token
        reqUrl = endpoint + Config.GetServers_Rest_URL
        httpClient = httplib.HTTPConnection(Config.OpenStack_URL, Config.MonitoringService_Port, Config.Request_Timeout)

        try:
            httpClient.request("GET", reqUrl, headers=headers)
            response = httpClient.getresponse()
            status =  response.status
            if response.status == 200:
                body = json.loads(response.read())
                if body:
                    servers = body.get('servers')
                    for server in servers:
                        resourceIdList.append(server.get("id"))
        except Exception, e:
            print e.message
        else:
            pass
        finally:
            if httpClient:
                httpClient.close();
        if status == 200:
            return resourceIdList
        elif status != 200 and retries > 1:
            print '####taotao retries'
            return self.getVms(token, endpoint,retries=retries - 1)
        else:
            return None

    def getMetricsAllParallel(self, base_data, token):
        # 取虚机列表
        vms = self.getVms(token.get(Config.OPENSTACK_SERVICE_TOKEN),
                          token.get(Config.OPENSTACK_SERVICE_NOVA))

        jobs = [gevent.spawn(self.getSample, Config.METER_NAME_CPU, resource_id,
                             token.get(Config.OPENSTACK_SERVICE_TOKEN),
                             token.get(Config.OPENSTACK_SERVICE_CEILOMETER), resource_id) for resource_id in vms]

        jobs += [gevent.spawn(self.getSample, Config.METER_NAME_MEM, resource_id,
                              token.get(Config.OPENSTACK_SERVICE_TOKEN),
                              token.get(Config.OPENSTACK_SERVICE_CEILOMETER), resource_id) for resource_id in vms]

        for job in base_data:
            for vm_id, meter_dict in job.iteritems():
                for meter_name, objs in meter_dict.iteritems():
                    if meter_name == 'interfaces':
                        jobs += [gevent.spawn(self.getSample, Config.METER_NAME_NETWORK_OUTGOING, nic,
                                              token.get(Config.OPENSTACK_SERVICE_TOKEN),
                                              token.get(Config.OPENSTACK_SERVICE_CEILOMETER), vm_id)
                                 for nic in objs]
                        #gevent.sleep(5)
                        jobs += [gevent.spawn(self.getSample, Config.METER_NAME_NETWORK_INCOMING, nic,
                                              token.get(Config.OPENSTACK_SERVICE_TOKEN),
                                              token.get(Config.OPENSTACK_SERVICE_CEILOMETER), vm_id)
                                 for nic in objs]

                    elif meter_name == 'disks':
                        jobs += [gevent.spawn(self.getSample, Config.METER_NAME_DISK_READ, volume,
                                              token.get(Config.OPENSTACK_SERVICE_TOKEN),
                                              token.get(Config.OPENSTACK_SERVICE_CEILOMETER), vm_id)
                                 for volume in objs]
                        #gevent.sleep(5)
                        jobs += [gevent.spawn(self.getSample, Config.METER_NAME_DISK_WRITE, volume,
                                              token.get(Config.OPENSTACK_SERVICE_TOKEN),
                                              token.get(Config.OPENSTACK_SERVICE_CEILOMETER), vm_id)
                                 for volume in objs]

        jobs += [gevent.spawn(self.getAlarmNums, resource_id)  for resource_id in vms]
        gevent.joinall(jobs, timeout=180)
        return [job.value for job in jobs]

    def getMetricsCpuAndMemory(self, token):
        # 取虚机列表
        vms = self.getVms(token.get(Config.OPENSTACK_SERVICE_TOKEN),
                          token.get(Config.OPENSTACK_SERVICE_NOVA))

        jobs = [gevent.spawn(self.getSample, Config.METER_NAME_CPU, resource_id,
                             token.get(Config.OPENSTACK_SERVICE_TOKEN),
                             token.get(Config.OPENSTACK_SERVICE_CEILOMETER), resource_id) for resource_id in vms]

        jobs += [gevent.spawn(self.getSample, Config.METER_NAME_MEM, resource_id,
                              token.get(Config.OPENSTACK_SERVICE_TOKEN),
                              token.get(Config.OPENSTACK_SERVICE_CEILOMETER), resource_id) for resource_id in vms]
        jobs += [gevent.spawn(self.getAlarmNums, resource_id) for resource_id in vms]

        gevent.joinall(jobs, timeout=180)
        return [job.value for job in jobs]

    def getMetricsNetworkOut(self, base_data, token):
        for job in base_data:
            for vm_id, meter_dict in job.iteritems():
                for meter_name, objs in meter_dict.iteritems():
                    if meter_name == 'interfaces':
                        jobs = [gevent.spawn(self.getSample, Config.METER_NAME_NETWORK_OUTGOING, nic,
                                              token.get(Config.OPENSTACK_SERVICE_TOKEN),
                                              token.get(Config.OPENSTACK_SERVICE_CEILOMETER), vm_id)
                                 for nic in objs]

        gevent.joinall(jobs, timeout=180)
        return [job.value for job in jobs]

    def getMetricsNetworkIn(self, base_data, token):
        for job in base_data:
            for vm_id, meter_dict in job.iteritems():
                for meter_name, objs in meter_dict.iteritems():
                    if meter_name == 'interfaces':
                        jobs = [gevent.spawn(self.getSample, Config.METER_NAME_NETWORK_INCOMING, nic,
                                              token.get(Config.OPENSTACK_SERVICE_TOKEN),
                                              token.get(Config.OPENSTACK_SERVICE_CEILOMETER), vm_id)
                                 for nic in objs]

        gevent.joinall(jobs, timeout=180)
        return [job.value for job in jobs]

    def getMetricsDisksRead(self, base_data, token):
        for job in base_data:
            for vm_id, meter_dict in job.iteritems():
                for meter_name, objs in meter_dict.iteritems():
                    if meter_name == 'disks':
                        jobs = [gevent.spawn(self.getSample, Config.METER_NAME_DISK_READ, volume,
                                              token.get(Config.OPENSTACK_SERVICE_TOKEN),
                                              token.get(Config.OPENSTACK_SERVICE_CEILOMETER), vm_id)
                                 for volume in objs]

        gevent.joinall(jobs, timeout=180)
        return [job.value for job in jobs]

    def getMetricsDisksWrite(self, base_data, token):
        for job in base_data:
            for vm_id, meter_dict in job.iteritems():
                for meter_name, objs in meter_dict.iteritems():
                    if meter_name == 'disks':
                        jobs = [gevent.spawn(self.getSample, Config.METER_NAME_DISK_WRITE, volume,
                                              token.get(Config.OPENSTACK_SERVICE_TOKEN),
                                              token.get(Config.OPENSTACK_SERVICE_CEILOMETER), vm_id)
                                 for volume in objs]

        gevent.joinall(jobs, timeout=180)
        return [job.value for job in jobs]

    def getBaseData(self, token):
        # 取虚机列表
        vms = self.getVms(token.get(Config.OPENSTACK_SERVICE_TOKEN),
                          token.get(Config.OPENSTACK_SERVICE_NOVA))
        # 取虚机对应的网卡列表
        jobs = [gevent.spawn(self.getInterfaces, resource_id, token.get(Config.OPENSTACK_SERVICE_TOKEN),
                             token.get(Config.OPENSTACK_SERVICE_NOVA)) for resource_id in vms]

        # 取虚机对应的磁盘列表
        jobs += [gevent.spawn(self.getVolumes, resource_id, token.get(Config.OPENSTACK_SERVICE_TOKEN),
                              token.get(Config.OPENSTACK_SERVICE_NOVA)) for resource_id in vms]

        gevent.joinall(jobs, timeout=60)
        return [job.value for job in jobs]


    def getMetricsRest(self, resource_id):
        meters = {
            'resource_id': "",
            'cpu_util': '0.00',
            'memory_usage': '0.00',
            'disk_read_bytes_rate': '0.00',
            'disk_write_bytes_rate': '0.00',
            'network_incoming_bytes_rate': '0.00',
            'network_outgoing_bytes_rate': '0.00',
            'alarm_nums': '-1'
        }

        cm = session.query(CeilometerMetrics).filter_by(nedn=resource_id).order_by(CeilometerMetrics.begintime.desc()).first()
        session.commit()
        meters['resource_id'] = resource_id
        if not cm:
            meters['cpu_util'] = cm.cpuusageatio
            meters['memory_usage'] = cm.memoryusageratio
            meters['disk_read_bytes_rate'] = cm.diskreadbytes
            meters['disk_write_bytes_rate'] = cm.diskwritebytes
            meters['network_incoming_bytes_rate'] = cm.networkincomingbytes
            meters['network_outgoing_bytes_rate'] = cm.networkoutgoingbytes
            meters['alarm_nums'] = cm.alarmnums

        return meters

    '''
    合并数据成如下格式:
    {'resource_id': [{'memory.usage': 71.58}, {'cpu_util': 60}, {'network.incoming.bytes.rate': 4534}, {'network.outgoing.bytes.rate': 32534}, {'network.outgoing.bytes.rate': 234}]
    '''
    def union_data(self, meters):
        ret = {}
        for meter in meters:
            if meter:
                for k, v in meter.iteritems():
                    if k in ret.keys():
                        ret[k] = ret[k] + [v]
                    else:
                        ret[k] = [v]
        return ret

    '''
    将union_data函数的返回结果，聚合成如下格式：
    {'resource_id': {'cpu_util': 120, 'network.incoming.bytes.rate': 16658, 'network.outgoing.bytes.rate': 32768, 'memory.usage': 71.58}]
    '''
    def aggregate_data(self, meters):
        ret_data = {}
        for res, meter_list in meters.iteritems():
            ret = {}
            for val in meter_list:
                for k, v in val.iteritems():
                    if k in ret.keys():
                        ret[k] = ret[k] + v
                    else:
                        ret[k] = v
            ret_data[res] = ret
        return ret_data

    def storeMetrics(self, aggregate_data):
        for resource_id, meters in aggregate_data.iteritems():
            nedn = resource_id
            granularity = 480
            begintime = '2017-10-01T08:52:01.392000'
            endtime = '2017-10-01T08:59:01.392000'
            cpuusageatio = None
            memoryusageratio = None
            diskreadbytes = None
            diskwritebytes = None
            networkincomingbytes = None
            networkoutgoingbytes = None
            alarmnums = None
            for meter_name, meter_value in meters.iteritems():
                if meter_name == Config.METER_NAME_CPU:
                    cpuusageatio = meters.get(Config.METER_NAME_CPU)
                elif meter_name == Config.METER_NAME_MEM:
                    memoryusageratio = meters.get(Config.METER_NAME_MEM)
                elif meter_name == Config.METER_NAME_DISK_READ:
                    diskreadbytes = meters.get(Config.METER_NAME_DISK_READ)
                elif meter_name == Config.METER_NAME_DISK_WRITE:
                    diskwritebytes = meters.get(Config.METER_NAME_DISK_WRITE)
                elif meter_name == Config.METER_NAME_NETWORK_INCOMING:
                    networkincomingbytes = meters.get(Config.METER_NAME_NETWORK_INCOMING)
                elif meter_name == Config.METER_NAME_NETWORK_OUTGOING:
                    networkoutgoingbytes = meters.get(Config.METER_NAME_NETWORK_OUTGOING)
                elif meter_name == Config.METER_ALARM_NUMS:
                    alarmnums = meters.get(Config.METER_ALARM_NUMS)
                elif meter_name == Config.METER_TIMESTAMP:
                    if len(meters.get(Config.METER_TIMESTAMP)) != 26:
                        meters[Config.METER_TIMESTAMP] = meters.get(Config.METER_TIMESTAMP) + '.000000'
                    begintime = datetime.datetime.strptime(meters.get(Config.METER_TIMESTAMP), '%Y-%m-%dT%H:%M:%S.%f')
                    endtime = (begintime + datetime.timedelta(minutes=7)).strftime('%Y-%m-%d %H:%M:%S.%f')

            met = session.query(CeilometerMetrics).filter(CeilometerMetrics.nedn==nedn).filter(CeilometerMetrics.granularity==granularity).first()
            if met:
                session.query(CeilometerMetrics).filter(CeilometerMetrics.nedn == nedn).filter(CeilometerMetrics.granularity == granularity).\
                    update({CeilometerMetrics.begintime:begintime,
                        CeilometerMetrics.endtime:endtime,
                        CeilometerMetrics.cpuusageatio:cpuusageatio,
                        CeilometerMetrics.memoryusageratio:memoryusageratio,
                        CeilometerMetrics.diskreadbytes:diskreadbytes,
                        CeilometerMetrics.diskwritebytes:diskwritebytes,
                        CeilometerMetrics.networkincomingbytes:networkincomingbytes,
                        CeilometerMetrics.networkoutgoingbytes:networkoutgoingbytes,
                        CeilometerMetrics.alarmnums:alarmnums})
            else:
                cm = CeilometerMetrics(nedn, granularity, begintime, endtime, cpuusageatio, memoryusageratio,
                                               diskreadbytes, diskwritebytes, networkincomingbytes, networkoutgoingbytes, alarmnums)
                session.add(cm)
        session.commit()

    def startProcess(self):
        token = Identity().getToken()
        mm = MeterManagerParallel()
        base_data = mm.getBaseData(token)
        if base_data.count(None) == 0:
            print "get base data success"
        else:
            print "get base data fail"
        data = mm.getMetricsAllParallel(base_data, token)
        if data.count(None) == 0:
            print "get data success"
        else:
            print "get data fail"

        union_data = mm.union_data(data)
        aggregate_data = mm.aggregate_data(union_data)
        mm.storeMetrics(aggregate_data)

    def getMetricsRest(self, resource_id):
        meters = {
            'resource_id': "",
            "cpu_util": '0.00',
            'memory_usage': '0.00',
            'disk_read_bytes_rate': '0.00',
            'disk_write_bytes_rate': '0.00',
            'network_incoming_bytes_rate': '0.00',
            'network_outgoing_bytes_rate': '0.00',
            'alarm_nums': '-1'
        }

        cm = session.query(CeilometerMetrics).filter_by(nedn=resource_id).order_by(
            CeilometerMetrics.begintime.desc()).first()

        meters['resource_id'] = resource_id
        if cm:
            meters["cpu_util"] = str(cm.cpuusageatio)
            meters['memory_usage'] = str(cm.memoryusageratio)
            meters['disk_read_bytes_rate'] = str(cm.diskreadbytes)
            meters['disk_write_bytes_rate'] = str(cm.diskwritebytes)
            meters['network_incoming_bytes_rate'] = str(cm.networkincomingbytes)
            meters['network_outgoing_bytes_rate'] = str(cm.networkoutgoingbytes)
            meters['alarm_nums'] = str(cm.alarmnums)

        return meters


